{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pickle文件读取完毕\n",
      "CPU times: user 5.39 s, sys: 8.83 s, total: 14.2 s\n",
      "Wall time: 14.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#刚才wb是写文件 这里的rb就是读文件\n",
    "with open(\"./train.pkl\",'rb') as f:\n",
    "    train=pickle.load(f)\n",
    "\n",
    "with open(\"./test.pkl\",'rb') as f:\n",
    "    test=pickle.load(f)\n",
    "print(\"pickle文件读取完毕\")    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### file_id的意思是文件编号   \n",
    "##### label的意思是标签 分为8个类别 0-正常/1-勒索病毒/2-挖矿程序/3-DDoS木马/4-蠕虫病毒/5-感染型病毒/6-后门程序/7-木马程序\n",
    "##### api 文件调用的api名称\n",
    "##### index 线程中API调用的顺序编号"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_id</th>\n",
       "      <th>label</th>\n",
       "      <th>api</th>\n",
       "      <th>tid</th>\n",
       "      <th>index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>LdrLoadDll</td>\n",
       "      <td>2488</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>LdrGetProcedureAddress</td>\n",
       "      <td>2488</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>LdrGetProcedureAddress</td>\n",
       "      <td>2488</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>LdrGetProcedureAddress</td>\n",
       "      <td>2488</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>LdrGetProcedureAddress</td>\n",
       "      <td>2488</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89806688</th>\n",
       "      <td>13887</td>\n",
       "      <td>2</td>\n",
       "      <td>NtClose</td>\n",
       "      <td>2336</td>\n",
       "      <td>618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89806689</th>\n",
       "      <td>13887</td>\n",
       "      <td>2</td>\n",
       "      <td>NtClose</td>\n",
       "      <td>2336</td>\n",
       "      <td>619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89806690</th>\n",
       "      <td>13887</td>\n",
       "      <td>2</td>\n",
       "      <td>NtClose</td>\n",
       "      <td>2336</td>\n",
       "      <td>620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89806691</th>\n",
       "      <td>13887</td>\n",
       "      <td>2</td>\n",
       "      <td>NtClose</td>\n",
       "      <td>2336</td>\n",
       "      <td>621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89806692</th>\n",
       "      <td>13887</td>\n",
       "      <td>2</td>\n",
       "      <td>NtTerminateProcess</td>\n",
       "      <td>2336</td>\n",
       "      <td>622</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>89806693 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          file_id  label                     api   tid  index\n",
       "0               1      5              LdrLoadDll  2488      0\n",
       "1               1      5  LdrGetProcedureAddress  2488      1\n",
       "2               1      5  LdrGetProcedureAddress  2488      2\n",
       "3               1      5  LdrGetProcedureAddress  2488      3\n",
       "4               1      5  LdrGetProcedureAddress  2488      4\n",
       "...           ...    ...                     ...   ...    ...\n",
       "89806688    13887      2                 NtClose  2336    618\n",
       "89806689    13887      2                 NtClose  2336    619\n",
       "89806690    13887      2                 NtClose  2336    620\n",
       "89806691    13887      2                 NtClose  2336    621\n",
       "89806692    13887      2      NtTerminateProcess  2336    622\n",
       "\n",
       "[89806693 rows x 5 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train#file_id的意思是文件编号   label的意思是标签 分为8个类别 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_id</th>\n",
       "      <th>api</th>\n",
       "      <th>tid</th>\n",
       "      <th>index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>RegOpenKeyExA</td>\n",
       "      <td>2332</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>CopyFileA</td>\n",
       "      <td>2332</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>OpenSCManagerA</td>\n",
       "      <td>2332</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>CreateServiceA</td>\n",
       "      <td>2332</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>RegOpenKeyExA</td>\n",
       "      <td>2468</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79288370</th>\n",
       "      <td>12955</td>\n",
       "      <td>Thread32Next</td>\n",
       "      <td>2740</td>\n",
       "      <td>1446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79288371</th>\n",
       "      <td>12955</td>\n",
       "      <td>Thread32Next</td>\n",
       "      <td>2740</td>\n",
       "      <td>1447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79288372</th>\n",
       "      <td>12955</td>\n",
       "      <td>NtClose</td>\n",
       "      <td>2740</td>\n",
       "      <td>1448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79288373</th>\n",
       "      <td>12955</td>\n",
       "      <td>__exception__</td>\n",
       "      <td>2740</td>\n",
       "      <td>1449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79288374</th>\n",
       "      <td>12955</td>\n",
       "      <td>NtTerminateProcess</td>\n",
       "      <td>2740</td>\n",
       "      <td>1450</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>79288375 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          file_id                 api   tid  index\n",
       "0               1       RegOpenKeyExA  2332      0\n",
       "1               1           CopyFileA  2332      1\n",
       "2               1      OpenSCManagerA  2332      2\n",
       "3               1      CreateServiceA  2332      3\n",
       "4               1       RegOpenKeyExA  2468      0\n",
       "...           ...                 ...   ...    ...\n",
       "79288370    12955        Thread32Next  2740   1446\n",
       "79288371    12955        Thread32Next  2740   1447\n",
       "79288372    12955             NtClose  2740   1448\n",
       "79288373    12955       __exception__  2740   1449\n",
       "79288374    12955  NtTerminateProcess  2740   1450\n",
       "\n",
       "[79288375 rows x 4 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 89806693 entries, 0 to 89806692\n",
      "Data columns (total 5 columns):\n",
      " #   Column   Dtype \n",
      "---  ------   ----- \n",
      " 0   file_id  int64 \n",
      " 1   label    int64 \n",
      " 2   api      object\n",
      " 3   tid      int64 \n",
      " 4   index    int64 \n",
      "dtypes: int64(4), object(1)\n",
      "memory usage: 3.3+ GB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10811    888204\n",
       "11303    791687\n",
       "5885     537495\n",
       "13468    506888\n",
       "9540     443957\n",
       "          ...  \n",
       "12736         1\n",
       "1253          1\n",
       "7871          1\n",
       "6643          1\n",
       "6060          1\n",
       "Name: file_id, Length: 13887, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 数据已经读取完毕了  开始数据探索一下\n",
    "train['file_id'].value_counts()#文件编号一共是13887个  9千万的数据里面 file_id一共是1万3000多个  也就是有1万3千个file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count                   89806693\n",
       "unique                       295\n",
       "top       LdrGetProcedureAddress\n",
       "freq                    10704305\n",
       "Name: api, dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['api'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LdrGetProcedureAddress    10704305\n",
       "NtClose                    6584708\n",
       "RegQueryValueExW           5506443\n",
       "Thread32Next               5321951\n",
       "NtDelayExecution           4360478\n",
       "                            ...   \n",
       "ExitWindowsEx                    2\n",
       "WSASendTo                        2\n",
       "NetUserGetLocalGroups            2\n",
       "NtUnloadDriver                   1\n",
       "EncryptMessage                   1\n",
       "Name: api, Length: 295, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['api'].value_counts()#这个是个object类型的  下一步转成lableencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LabelEncoder()"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "le=LabelEncoder()\n",
    "le#调包已经调好了  不过下一步需要把测试集和训练集都进行一下这个操作  按照原来的方法 是先对train进行操作 然后再对test进行操作\n",
    "#不过我们这里可以用一个新的方法 那就是将train和test先进行合并 然后再一起操作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_id</th>\n",
       "      <th>label</th>\n",
       "      <th>api</th>\n",
       "      <th>tid</th>\n",
       "      <th>index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>5.0</td>\n",
       "      <td>LdrLoadDll</td>\n",
       "      <td>2488</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>5.0</td>\n",
       "      <td>LdrGetProcedureAddress</td>\n",
       "      <td>2488</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>5.0</td>\n",
       "      <td>LdrGetProcedureAddress</td>\n",
       "      <td>2488</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>5.0</td>\n",
       "      <td>LdrGetProcedureAddress</td>\n",
       "      <td>2488</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>5.0</td>\n",
       "      <td>LdrGetProcedureAddress</td>\n",
       "      <td>2488</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79288370</th>\n",
       "      <td>12955</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Thread32Next</td>\n",
       "      <td>2740</td>\n",
       "      <td>1446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79288371</th>\n",
       "      <td>12955</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Thread32Next</td>\n",
       "      <td>2740</td>\n",
       "      <td>1447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79288372</th>\n",
       "      <td>12955</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NtClose</td>\n",
       "      <td>2740</td>\n",
       "      <td>1448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79288373</th>\n",
       "      <td>12955</td>\n",
       "      <td>NaN</td>\n",
       "      <td>__exception__</td>\n",
       "      <td>2740</td>\n",
       "      <td>1449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79288374</th>\n",
       "      <td>12955</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NtTerminateProcess</td>\n",
       "      <td>2740</td>\n",
       "      <td>1450</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>169095068 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          file_id  label                     api   tid  index\n",
       "0               1    5.0              LdrLoadDll  2488      0\n",
       "1               1    5.0  LdrGetProcedureAddress  2488      1\n",
       "2               1    5.0  LdrGetProcedureAddress  2488      2\n",
       "3               1    5.0  LdrGetProcedureAddress  2488      3\n",
       "4               1    5.0  LdrGetProcedureAddress  2488      4\n",
       "...           ...    ...                     ...   ...    ...\n",
       "79288370    12955    NaN            Thread32Next  2740   1446\n",
       "79288371    12955    NaN            Thread32Next  2740   1447\n",
       "79288372    12955    NaN                 NtClose  2740   1448\n",
       "79288373    12955    NaN           __exception__  2740   1449\n",
       "79288374    12955    NaN      NtTerminateProcess  2740   1450\n",
       "\n",
       "[169095068 rows x 5 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#将训练集和测试集合并  这里合并的意图是这样的，因为labelEncoder的意思是做编号排序，为避免 测试集里面有一些测试集里面没有的 \n",
    "#这样labelEncoder就不能完整的编号  合并之后一切encoder 那么就没有这个问题了\n",
    "df_all=pd.concat([train,test])#concat是按照竖直合并\n",
    "df_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0           135\n",
       "1           134\n",
       "2           134\n",
       "3           134\n",
       "4           134\n",
       "           ... \n",
       "79288370    266\n",
       "79288371    266\n",
       "79288372    152\n",
       "79288373    281\n",
       "79288374    197\n",
       "Name: api, Length: 169095068, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all['api']=le.fit_transform(df_all['api'])\n",
    "df_all['api']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "134    20455492\n",
       "152    12209428\n",
       "231    10393562\n",
       "266    10320844\n",
       "159     8268423\n",
       "         ...   \n",
       "274           2\n",
       "264           1\n",
       "160           1\n",
       "49            1\n",
       "158           1\n",
       "Name: api, Length: 301, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all['api'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  0,   1,   2,   3,   4,   5,   6,   7,   8,   9,  10,  11,  12,\n",
       "        13,  14,  15,  16,  17,  18,  19,  20,  21,  22,  23,  24,  25,\n",
       "        26,  27,  28,  29,  30,  31,  32,  33,  34,  35,  36,  37,  38,\n",
       "        39,  40,  41,  42,  43,  44,  45,  46,  47,  48,  49,  50,  51,\n",
       "        52,  53,  54,  55,  56,  57,  58,  59,  60,  61,  62,  63,  64,\n",
       "        65,  66,  67,  68,  69,  70,  71,  72,  73,  74,  75,  76,  77,\n",
       "        78,  79,  80,  81,  82,  83,  84,  85,  86,  87,  88,  89,  90,\n",
       "        91,  92,  93,  94,  95,  96,  97,  98,  99, 100, 101, 102, 103,\n",
       "       104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116,\n",
       "       117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129,\n",
       "       130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142,\n",
       "       143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155,\n",
       "       156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168,\n",
       "       169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181,\n",
       "       182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194,\n",
       "       195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207,\n",
       "       208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220,\n",
       "       221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233,\n",
       "       234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246,\n",
       "       247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259,\n",
       "       260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272,\n",
       "       273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285,\n",
       "       286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298,\n",
       "       299, 300])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sort(df_all['api'].value_counts().index.tolist())#这里就是labelencoder转化为了顺序的数字 0-300 一共301个"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_id</th>\n",
       "      <th>label</th>\n",
       "      <th>api</th>\n",
       "      <th>tid</th>\n",
       "      <th>index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>135</td>\n",
       "      <td>2488</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>134</td>\n",
       "      <td>2488</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>134</td>\n",
       "      <td>2488</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>134</td>\n",
       "      <td>2488</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>134</td>\n",
       "      <td>2488</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89806688</th>\n",
       "      <td>13887</td>\n",
       "      <td>2</td>\n",
       "      <td>152</td>\n",
       "      <td>2336</td>\n",
       "      <td>618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89806689</th>\n",
       "      <td>13887</td>\n",
       "      <td>2</td>\n",
       "      <td>152</td>\n",
       "      <td>2336</td>\n",
       "      <td>619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89806690</th>\n",
       "      <td>13887</td>\n",
       "      <td>2</td>\n",
       "      <td>152</td>\n",
       "      <td>2336</td>\n",
       "      <td>620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89806691</th>\n",
       "      <td>13887</td>\n",
       "      <td>2</td>\n",
       "      <td>152</td>\n",
       "      <td>2336</td>\n",
       "      <td>621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89806692</th>\n",
       "      <td>13887</td>\n",
       "      <td>2</td>\n",
       "      <td>197</td>\n",
       "      <td>2336</td>\n",
       "      <td>622</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>89806693 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          file_id  label  api   tid  index\n",
       "0               1      5  135  2488      0\n",
       "1               1      5  134  2488      1\n",
       "2               1      5  134  2488      2\n",
       "3               1      5  134  2488      3\n",
       "4               1      5  134  2488      4\n",
       "...           ...    ...  ...   ...    ...\n",
       "89806688    13887      2  152  2336    618\n",
       "89806689    13887      2  152  2336    619\n",
       "89806690    13887      2  152  2336    620\n",
       "89806691    13887      2  152  2336    621\n",
       "89806692    13887      2  197  2336    622\n",
       "\n",
       "[89806693 rows x 5 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#这样都操作完了以后  再根据把train 和test进行分开  就是通过label那个字段操作  train是有值的 test是空的\n",
    "train['api']=df_all[df_all['label'].notnull()]['api']\n",
    "test['api']=df_all[df_all['label'].isnull()]['api']\n",
    "train#感觉这种方法好像也没有比单独对test和train操作省了多少力气 不过学会一种新的方法 技多不压身\n",
    "#之类的api这一列原来的很多字符都变成了 数字\n",
    "#API物理意义是文件调用的API的名称"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_id</th>\n",
       "      <th>api</th>\n",
       "      <th>tid</th>\n",
       "      <th>index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>226</td>\n",
       "      <td>2332</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>2332</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>205</td>\n",
       "      <td>2332</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>23</td>\n",
       "      <td>2332</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>226</td>\n",
       "      <td>2468</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79288370</th>\n",
       "      <td>12955</td>\n",
       "      <td>266</td>\n",
       "      <td>2740</td>\n",
       "      <td>1446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79288371</th>\n",
       "      <td>12955</td>\n",
       "      <td>266</td>\n",
       "      <td>2740</td>\n",
       "      <td>1447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79288372</th>\n",
       "      <td>12955</td>\n",
       "      <td>152</td>\n",
       "      <td>2740</td>\n",
       "      <td>1448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79288373</th>\n",
       "      <td>12955</td>\n",
       "      <td>281</td>\n",
       "      <td>2740</td>\n",
       "      <td>1449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79288374</th>\n",
       "      <td>12955</td>\n",
       "      <td>197</td>\n",
       "      <td>2740</td>\n",
       "      <td>1450</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>79288375 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          file_id  api   tid  index\n",
       "0               1  226  2332      0\n",
       "1               1   13  2332      1\n",
       "2               1  205  2332      2\n",
       "3               1   23  2332      3\n",
       "4               1  226  2468      0\n",
       "...           ...  ...   ...    ...\n",
       "79288370    12955  266  2740   1446\n",
       "79288371    12955  266  2740   1447\n",
       "79288372    12955  152  2740   1448\n",
       "79288373    12955  281  2740   1449\n",
       "79288374    12955  197  2740   1450\n",
       "\n",
       "[79288375 rows x 4 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 查看某个变量占用的资源情况"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7.559138625860214"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "sys.getsizeof(df_all)/1024/1024/1024#这里的单位是比特 除以1024是k 再除以1024是M  再除以1024是G  二者合起来7个G 差不多"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "######  释放内存 针对不用的变量可以把内存给释放 比如这里的df_all已经不用了  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "140"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gc\n",
    "del df_all\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df_all' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-4f8504db6899>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_all\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m#这里想用一下df_all已经不行了  因为已经删除了\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'df_all' is not defined"
     ]
    }
   ],
   "source": [
    "print(type(df_all))#这里想用一下df_all已经不行了  因为已经删除了"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 现在的问题是特征数量太少  需要构造新的特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['file_id', 'label', 'api', 'tid', 'index'], dtype='object') Index(['file_id', 'api', 'tid', 'index'], dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print(train.columns,test.columns)#可以看到 都只有4个特征  只有4个特征 无论什么模型 估计准确率都不会很高的"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### #没有聚合之前就是单纯的排列  聚合之后就是按照某一列的值把相同的求和了吧"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_id</th>\n",
       "      <th>label</th>\n",
       "      <th>api</th>\n",
       "      <th>tid</th>\n",
       "      <th>index</th>\n",
       "      <th>api_count</th>\n",
       "      <th>api_nunique</th>\n",
       "      <th>api_min</th>\n",
       "      <th>api_max</th>\n",
       "      <th>api_mean</th>\n",
       "      <th>...</th>\n",
       "      <th>tid_std</th>\n",
       "      <th>tid_ptp</th>\n",
       "      <th>index_count</th>\n",
       "      <th>index_nunique</th>\n",
       "      <th>index_min</th>\n",
       "      <th>index_max</th>\n",
       "      <th>index_mean</th>\n",
       "      <th>index_median</th>\n",
       "      <th>index_std</th>\n",
       "      <th>index_ptp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>135</td>\n",
       "      <td>2488</td>\n",
       "      <td>0</td>\n",
       "      <td>6786</td>\n",
       "      <td>116</td>\n",
       "      <td>6</td>\n",
       "      <td>298</td>\n",
       "      <td>171.965223</td>\n",
       "      <td>...</td>\n",
       "      <td>83.881299</td>\n",
       "      <td>324</td>\n",
       "      <td>6786</td>\n",
       "      <td>5001</td>\n",
       "      <td>0</td>\n",
       "      <td>5000</td>\n",
       "      <td>2000.806955</td>\n",
       "      <td>1607.5</td>\n",
       "      <td>1510.694221</td>\n",
       "      <td>5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6786</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>95</td>\n",
       "      <td>2320</td>\n",
       "      <td>0</td>\n",
       "      <td>816</td>\n",
       "      <td>30</td>\n",
       "      <td>89</td>\n",
       "      <td>298</td>\n",
       "      <td>159.696078</td>\n",
       "      <td>...</td>\n",
       "      <td>101.506783</td>\n",
       "      <td>284</td>\n",
       "      <td>816</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>203</td>\n",
       "      <td>101.500000</td>\n",
       "      <td>101.5</td>\n",
       "      <td>58.925137</td>\n",
       "      <td>203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7602</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>151</td>\n",
       "      <td>2208</td>\n",
       "      <td>0</td>\n",
       "      <td>463</td>\n",
       "      <td>42</td>\n",
       "      <td>9</td>\n",
       "      <td>258</td>\n",
       "      <td>164.948164</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>463</td>\n",
       "      <td>463</td>\n",
       "      <td>0</td>\n",
       "      <td>462</td>\n",
       "      <td>231.000000</td>\n",
       "      <td>231.0</td>\n",
       "      <td>133.800847</td>\n",
       "      <td>462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8065</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>95</td>\n",
       "      <td>2284</td>\n",
       "      <td>0</td>\n",
       "      <td>2046</td>\n",
       "      <td>51</td>\n",
       "      <td>9</td>\n",
       "      <td>257</td>\n",
       "      <td>154.939883</td>\n",
       "      <td>...</td>\n",
       "      <td>150.460506</td>\n",
       "      <td>696</td>\n",
       "      <td>2046</td>\n",
       "      <td>1028</td>\n",
       "      <td>0</td>\n",
       "      <td>1027</td>\n",
       "      <td>511.012219</td>\n",
       "      <td>511.0</td>\n",
       "      <td>295.407885</td>\n",
       "      <td>1027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10111</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>249</td>\n",
       "      <td>2500</td>\n",
       "      <td>0</td>\n",
       "      <td>10002</td>\n",
       "      <td>65</td>\n",
       "      <td>6</td>\n",
       "      <td>254</td>\n",
       "      <td>201.893421</td>\n",
       "      <td>...</td>\n",
       "      <td>49.556301</td>\n",
       "      <td>176</td>\n",
       "      <td>10002</td>\n",
       "      <td>5001</td>\n",
       "      <td>0</td>\n",
       "      <td>5000</td>\n",
       "      <td>2500.000000</td>\n",
       "      <td>2500.0</td>\n",
       "      <td>1443.736493</td>\n",
       "      <td>5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89620181</th>\n",
       "      <td>13883</td>\n",
       "      <td>2</td>\n",
       "      <td>95</td>\n",
       "      <td>100</td>\n",
       "      <td>0</td>\n",
       "      <td>178221</td>\n",
       "      <td>71</td>\n",
       "      <td>6</td>\n",
       "      <td>279</td>\n",
       "      <td>156.643100</td>\n",
       "      <td>...</td>\n",
       "      <td>1405.045515</td>\n",
       "      <td>6468</td>\n",
       "      <td>178221</td>\n",
       "      <td>5001</td>\n",
       "      <td>0</td>\n",
       "      <td>5000</td>\n",
       "      <td>401.480987</td>\n",
       "      <td>47.0</td>\n",
       "      <td>1008.636040</td>\n",
       "      <td>5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89798402</th>\n",
       "      <td>13884</td>\n",
       "      <td>5</td>\n",
       "      <td>95</td>\n",
       "      <td>2592</td>\n",
       "      <td>0</td>\n",
       "      <td>1319</td>\n",
       "      <td>39</td>\n",
       "      <td>6</td>\n",
       "      <td>279</td>\n",
       "      <td>163.025019</td>\n",
       "      <td>...</td>\n",
       "      <td>4.295386</td>\n",
       "      <td>156</td>\n",
       "      <td>1319</td>\n",
       "      <td>1319</td>\n",
       "      <td>0</td>\n",
       "      <td>1318</td>\n",
       "      <td>659.000000</td>\n",
       "      <td>659.0</td>\n",
       "      <td>380.906813</td>\n",
       "      <td>1318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89799721</th>\n",
       "      <td>13885</td>\n",
       "      <td>0</td>\n",
       "      <td>151</td>\n",
       "      <td>2240</td>\n",
       "      <td>0</td>\n",
       "      <td>1033</td>\n",
       "      <td>71</td>\n",
       "      <td>8</td>\n",
       "      <td>259</td>\n",
       "      <td>174.896418</td>\n",
       "      <td>...</td>\n",
       "      <td>33.152020</td>\n",
       "      <td>504</td>\n",
       "      <td>1033</td>\n",
       "      <td>1033</td>\n",
       "      <td>0</td>\n",
       "      <td>1032</td>\n",
       "      <td>516.000000</td>\n",
       "      <td>516.0</td>\n",
       "      <td>298.345717</td>\n",
       "      <td>1032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89800754</th>\n",
       "      <td>13886</td>\n",
       "      <td>1</td>\n",
       "      <td>95</td>\n",
       "      <td>2324</td>\n",
       "      <td>0</td>\n",
       "      <td>5316</td>\n",
       "      <td>80</td>\n",
       "      <td>9</td>\n",
       "      <td>281</td>\n",
       "      <td>168.313017</td>\n",
       "      <td>...</td>\n",
       "      <td>154.796790</td>\n",
       "      <td>512</td>\n",
       "      <td>5316</td>\n",
       "      <td>2503</td>\n",
       "      <td>0</td>\n",
       "      <td>2502</td>\n",
       "      <td>1173.050414</td>\n",
       "      <td>1165.5</td>\n",
       "      <td>755.545651</td>\n",
       "      <td>2502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89806070</th>\n",
       "      <td>13887</td>\n",
       "      <td>2</td>\n",
       "      <td>135</td>\n",
       "      <td>2336</td>\n",
       "      <td>0</td>\n",
       "      <td>623</td>\n",
       "      <td>37</td>\n",
       "      <td>11</td>\n",
       "      <td>277</td>\n",
       "      <td>139.784912</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>623</td>\n",
       "      <td>623</td>\n",
       "      <td>0</td>\n",
       "      <td>622</td>\n",
       "      <td>311.000000</td>\n",
       "      <td>311.0</td>\n",
       "      <td>179.988889</td>\n",
       "      <td>622</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13887 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          file_id  label  api   tid  index  api_count  api_nunique  api_min  \\\n",
       "0               1      5  135  2488      0       6786          116        6   \n",
       "6786            2      2   95  2320      0        816           30       89   \n",
       "7602            3      0  151  2208      0        463           42        9   \n",
       "8065            4      0   95  2284      0       2046           51        9   \n",
       "10111           5      0  249  2500      0      10002           65        6   \n",
       "...           ...    ...  ...   ...    ...        ...          ...      ...   \n",
       "89620181    13883      2   95   100      0     178221           71        6   \n",
       "89798402    13884      5   95  2592      0       1319           39        6   \n",
       "89799721    13885      0  151  2240      0       1033           71        8   \n",
       "89800754    13886      1   95  2324      0       5316           80        9   \n",
       "89806070    13887      2  135  2336      0        623           37       11   \n",
       "\n",
       "          api_max    api_mean  ...      tid_std  tid_ptp  index_count  \\\n",
       "0             298  171.965223  ...    83.881299      324         6786   \n",
       "6786          298  159.696078  ...   101.506783      284          816   \n",
       "7602          258  164.948164  ...     0.000000        0          463   \n",
       "8065          257  154.939883  ...   150.460506      696         2046   \n",
       "10111         254  201.893421  ...    49.556301      176        10002   \n",
       "...           ...         ...  ...          ...      ...          ...   \n",
       "89620181      279  156.643100  ...  1405.045515     6468       178221   \n",
       "89798402      279  163.025019  ...     4.295386      156         1319   \n",
       "89799721      259  174.896418  ...    33.152020      504         1033   \n",
       "89800754      281  168.313017  ...   154.796790      512         5316   \n",
       "89806070      277  139.784912  ...     0.000000        0          623   \n",
       "\n",
       "          index_nunique  index_min  index_max   index_mean  index_median  \\\n",
       "0                  5001          0       5000  2000.806955        1607.5   \n",
       "6786                204          0        203   101.500000         101.5   \n",
       "7602                463          0        462   231.000000         231.0   \n",
       "8065               1028          0       1027   511.012219         511.0   \n",
       "10111              5001          0       5000  2500.000000        2500.0   \n",
       "...                 ...        ...        ...          ...           ...   \n",
       "89620181           5001          0       5000   401.480987          47.0   \n",
       "89798402           1319          0       1318   659.000000         659.0   \n",
       "89799721           1033          0       1032   516.000000         516.0   \n",
       "89800754           2503          0       2502  1173.050414        1165.5   \n",
       "89806070            623          0        622   311.000000         311.0   \n",
       "\n",
       "            index_std  index_ptp  \n",
       "0         1510.694221       5000  \n",
       "6786        58.925137        203  \n",
       "7602       133.800847        462  \n",
       "8065       295.407885       1027  \n",
       "10111     1443.736493       5000  \n",
       "...               ...        ...  \n",
       "89620181  1008.636040       5000  \n",
       "89798402   380.906813       1318  \n",
       "89799721   298.345717       1032  \n",
       "89800754   755.545651       2502  \n",
       "89806070   179.988889        622  \n",
       "\n",
       "[13887 rows x 29 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## 按照file_id进行聚合统计\n",
    "def get_features(df):\n",
    "    \"\"\"传入一个df\"\"\"\n",
    "    df_file=df.groupby('file_id')#先按照file_id这个字段进行聚合\n",
    "    #df1为最终的结果\n",
    "    #训练集和测试集分开处理 先给这两个数据集去重 然后排序一下 之后再把上面按照 file_id聚合之后的数量特征复制给数据集\n",
    "    if 'label' in df.columns:#训练集\n",
    "        df1=df.drop_duplicates(subset=['file_id','label'],keep='first')#subset表示按照这一列进行去重 \n",
    "    else:\n",
    "        df1=df.drop_duplicates(subset=['file_id'],keep='first')#按照file_id字段进行去重 keep=first表示保留第一次出现的位置\n",
    "    df1=df1.sort_values('file_id')#按照file_id排序\n",
    "    #提取多个特征 统计特征['api','tid','index']\n",
    "    features=['api','tid','index']\n",
    "    #开始增加特征\n",
    "    for f in features:\n",
    "        ###这些常见的统计特征\n",
    "        df1[f+'_count']=df_file[f].count().values\n",
    "        df1[f+'_nunique']=df_file[f].nunique().values\n",
    "        df1[f+'_min']=df_file[f].min().values\n",
    "        df1[f+'_max']=df_file[f].max().values\n",
    "        df1[f+'_mean']=df_file[f].mean().values\n",
    "        df1[f+'_median']=df_file[f].median().values\n",
    "        df1[f+'_std']=df_file[f].std().values\n",
    "        df1[f+'_ptp']=df1[f+'_max']-df1[f+'_min']#最大值和最小值的差值\n",
    "        \n",
    "    return df1\n",
    "\n",
    "get_features(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_id</th>\n",
       "      <th>api</th>\n",
       "      <th>tid</th>\n",
       "      <th>index</th>\n",
       "      <th>api_count</th>\n",
       "      <th>api_nunique</th>\n",
       "      <th>api_min</th>\n",
       "      <th>api_max</th>\n",
       "      <th>api_mean</th>\n",
       "      <th>api_median</th>\n",
       "      <th>...</th>\n",
       "      <th>tid_std</th>\n",
       "      <th>tid_ptp</th>\n",
       "      <th>index_count</th>\n",
       "      <th>index_nunique</th>\n",
       "      <th>index_min</th>\n",
       "      <th>index_max</th>\n",
       "      <th>index_mean</th>\n",
       "      <th>index_median</th>\n",
       "      <th>index_std</th>\n",
       "      <th>index_ptp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>226</td>\n",
       "      <td>2332</td>\n",
       "      <td>0</td>\n",
       "      <td>97</td>\n",
       "      <td>15</td>\n",
       "      <td>13</td>\n",
       "      <td>262</td>\n",
       "      <td>155.989691</td>\n",
       "      <td>152.0</td>\n",
       "      <td>...</td>\n",
       "      <td>57.218548</td>\n",
       "      <td>236</td>\n",
       "      <td>97</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>14.443299</td>\n",
       "      <td>14.0</td>\n",
       "      <td>9.210466</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>2</td>\n",
       "      <td>226</td>\n",
       "      <td>2472</td>\n",
       "      <td>0</td>\n",
       "      <td>1361</td>\n",
       "      <td>40</td>\n",
       "      <td>6</td>\n",
       "      <td>261</td>\n",
       "      <td>138.025716</td>\n",
       "      <td>138.0</td>\n",
       "      <td>...</td>\n",
       "      <td>104.399149</td>\n",
       "      <td>276</td>\n",
       "      <td>1361</td>\n",
       "      <td>681</td>\n",
       "      <td>0</td>\n",
       "      <td>680</td>\n",
       "      <td>339.750184</td>\n",
       "      <td>340.0</td>\n",
       "      <td>196.515744</td>\n",
       "      <td>680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1458</th>\n",
       "      <td>3</td>\n",
       "      <td>95</td>\n",
       "      <td>2344</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>9</td>\n",
       "      <td>16</td>\n",
       "      <td>257</td>\n",
       "      <td>111.375000</td>\n",
       "      <td>134.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>7.500000</td>\n",
       "      <td>7.5</td>\n",
       "      <td>4.760952</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1474</th>\n",
       "      <td>4</td>\n",
       "      <td>135</td>\n",
       "      <td>2452</td>\n",
       "      <td>0</td>\n",
       "      <td>193</td>\n",
       "      <td>34</td>\n",
       "      <td>13</td>\n",
       "      <td>262</td>\n",
       "      <td>172.217617</td>\n",
       "      <td>170.0</td>\n",
       "      <td>...</td>\n",
       "      <td>50.951508</td>\n",
       "      <td>132</td>\n",
       "      <td>193</td>\n",
       "      <td>193</td>\n",
       "      <td>0</td>\n",
       "      <td>192</td>\n",
       "      <td>96.000000</td>\n",
       "      <td>96.0</td>\n",
       "      <td>55.858452</td>\n",
       "      <td>192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1667</th>\n",
       "      <td>5</td>\n",
       "      <td>95</td>\n",
       "      <td>2332</td>\n",
       "      <td>0</td>\n",
       "      <td>803</td>\n",
       "      <td>34</td>\n",
       "      <td>16</td>\n",
       "      <td>261</td>\n",
       "      <td>168.490660</td>\n",
       "      <td>153.0</td>\n",
       "      <td>...</td>\n",
       "      <td>201.826813</td>\n",
       "      <td>448</td>\n",
       "      <td>803</td>\n",
       "      <td>268</td>\n",
       "      <td>0</td>\n",
       "      <td>267</td>\n",
       "      <td>133.333748</td>\n",
       "      <td>133.0</td>\n",
       "      <td>77.317048</td>\n",
       "      <td>267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79277890</th>\n",
       "      <td>12951</td>\n",
       "      <td>151</td>\n",
       "      <td>2644</td>\n",
       "      <td>0</td>\n",
       "      <td>289</td>\n",
       "      <td>37</td>\n",
       "      <td>9</td>\n",
       "      <td>269</td>\n",
       "      <td>140.536332</td>\n",
       "      <td>151.0</td>\n",
       "      <td>...</td>\n",
       "      <td>75.402526</td>\n",
       "      <td>336</td>\n",
       "      <td>289</td>\n",
       "      <td>145</td>\n",
       "      <td>0</td>\n",
       "      <td>144</td>\n",
       "      <td>71.750865</td>\n",
       "      <td>72.0</td>\n",
       "      <td>41.786414</td>\n",
       "      <td>144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79278179</th>\n",
       "      <td>12952</td>\n",
       "      <td>151</td>\n",
       "      <td>2264</td>\n",
       "      <td>0</td>\n",
       "      <td>112</td>\n",
       "      <td>28</td>\n",
       "      <td>56</td>\n",
       "      <td>261</td>\n",
       "      <td>163.669643</td>\n",
       "      <td>152.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>112</td>\n",
       "      <td>112</td>\n",
       "      <td>0</td>\n",
       "      <td>111</td>\n",
       "      <td>55.500000</td>\n",
       "      <td>55.5</td>\n",
       "      <td>32.475632</td>\n",
       "      <td>111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79278291</th>\n",
       "      <td>12953</td>\n",
       "      <td>135</td>\n",
       "      <td>2324</td>\n",
       "      <td>0</td>\n",
       "      <td>5095</td>\n",
       "      <td>72</td>\n",
       "      <td>6</td>\n",
       "      <td>286</td>\n",
       "      <td>200.063199</td>\n",
       "      <td>214.0</td>\n",
       "      <td>...</td>\n",
       "      <td>196.695730</td>\n",
       "      <td>560</td>\n",
       "      <td>5095</td>\n",
       "      <td>1464</td>\n",
       "      <td>0</td>\n",
       "      <td>1463</td>\n",
       "      <td>538.423749</td>\n",
       "      <td>454.0</td>\n",
       "      <td>393.605016</td>\n",
       "      <td>1463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79283386</th>\n",
       "      <td>12954</td>\n",
       "      <td>135</td>\n",
       "      <td>2424</td>\n",
       "      <td>0</td>\n",
       "      <td>2951</td>\n",
       "      <td>65</td>\n",
       "      <td>9</td>\n",
       "      <td>298</td>\n",
       "      <td>191.007794</td>\n",
       "      <td>139.0</td>\n",
       "      <td>...</td>\n",
       "      <td>126.124152</td>\n",
       "      <td>276</td>\n",
       "      <td>2951</td>\n",
       "      <td>1445</td>\n",
       "      <td>0</td>\n",
       "      <td>1444</td>\n",
       "      <td>596.701796</td>\n",
       "      <td>555.0</td>\n",
       "      <td>397.358069</td>\n",
       "      <td>1444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79286337</th>\n",
       "      <td>12955</td>\n",
       "      <td>135</td>\n",
       "      <td>2500</td>\n",
       "      <td>0</td>\n",
       "      <td>2038</td>\n",
       "      <td>54</td>\n",
       "      <td>13</td>\n",
       "      <td>284</td>\n",
       "      <td>208.845927</td>\n",
       "      <td>266.0</td>\n",
       "      <td>...</td>\n",
       "      <td>78.912837</td>\n",
       "      <td>240</td>\n",
       "      <td>2038</td>\n",
       "      <td>1451</td>\n",
       "      <td>0</td>\n",
       "      <td>1450</td>\n",
       "      <td>560.742885</td>\n",
       "      <td>431.5</td>\n",
       "      <td>440.983364</td>\n",
       "      <td>1450</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12955 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          file_id  api   tid  index  api_count  api_nunique  api_min  api_max  \\\n",
       "0               1  226  2332      0         97           15       13      262   \n",
       "97              2  226  2472      0       1361           40        6      261   \n",
       "1458            3   95  2344      0         16            9       16      257   \n",
       "1474            4  135  2452      0        193           34       13      262   \n",
       "1667            5   95  2332      0        803           34       16      261   \n",
       "...           ...  ...   ...    ...        ...          ...      ...      ...   \n",
       "79277890    12951  151  2644      0        289           37        9      269   \n",
       "79278179    12952  151  2264      0        112           28       56      261   \n",
       "79278291    12953  135  2324      0       5095           72        6      286   \n",
       "79283386    12954  135  2424      0       2951           65        9      298   \n",
       "79286337    12955  135  2500      0       2038           54       13      284   \n",
       "\n",
       "            api_mean  api_median  ...     tid_std  tid_ptp  index_count  \\\n",
       "0         155.989691       152.0  ...   57.218548      236           97   \n",
       "97        138.025716       138.0  ...  104.399149      276         1361   \n",
       "1458      111.375000       134.0  ...    0.000000        0           16   \n",
       "1474      172.217617       170.0  ...   50.951508      132          193   \n",
       "1667      168.490660       153.0  ...  201.826813      448          803   \n",
       "...              ...         ...  ...         ...      ...          ...   \n",
       "79277890  140.536332       151.0  ...   75.402526      336          289   \n",
       "79278179  163.669643       152.0  ...    0.000000        0          112   \n",
       "79278291  200.063199       214.0  ...  196.695730      560         5095   \n",
       "79283386  191.007794       139.0  ...  126.124152      276         2951   \n",
       "79286337  208.845927       266.0  ...   78.912837      240         2038   \n",
       "\n",
       "          index_nunique  index_min  index_max  index_mean  index_median  \\\n",
       "0                    31          0         30   14.443299          14.0   \n",
       "97                  681          0        680  339.750184         340.0   \n",
       "1458                 16          0         15    7.500000           7.5   \n",
       "1474                193          0        192   96.000000          96.0   \n",
       "1667                268          0        267  133.333748         133.0   \n",
       "...                 ...        ...        ...         ...           ...   \n",
       "79277890            145          0        144   71.750865          72.0   \n",
       "79278179            112          0        111   55.500000          55.5   \n",
       "79278291           1464          0       1463  538.423749         454.0   \n",
       "79283386           1445          0       1444  596.701796         555.0   \n",
       "79286337           1451          0       1450  560.742885         431.5   \n",
       "\n",
       "           index_std  index_ptp  \n",
       "0           9.210466         30  \n",
       "97        196.515744        680  \n",
       "1458        4.760952         15  \n",
       "1474       55.858452        192  \n",
       "1667       77.317048        267  \n",
       "...              ...        ...  \n",
       "79277890   41.786414        144  \n",
       "79278179   32.475632        111  \n",
       "79278291  393.605016       1463  \n",
       "79283386  397.358069       1444  \n",
       "79286337  440.983364       1450  \n",
       "\n",
       "[12955 rows x 28 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train=get_features(train)\n",
    "df_test=get_features(test)\n",
    "df_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 保存为pkl文件 这下就很小了"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#保存为格式较小的pickle文件  这下就只有几兆了\n",
    "df_train.to_pickle('./df_train.pkl')\n",
    "df_test.to_pickle('./df_test.pkl')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
